import  requests,json
import time,hashlib,re,pymysql
from lxml import etree





class toutiao():
    def __init__(self):
        print('开始配置')
        city_url='https://www.toutiao.com/stream/widget/local_weather/city/'
        self.headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
                 'Host':'www.toutiao.com',
                 'Referer':'https://www.toutiao.com/',
                 'Cookie':'''UM_distinctid=15f32dd7ecca02-0ffd47e89664fe-3b3e5906-1fa400-15f32dd7ecdd9a; uuid="w:7547587fcc234d0e95fd3af0612114de"; tt_webid=6478472404099532302; CNZZDATA1259612802=556198009-1508382043-https%253A%252F%252Fwww.baidu.com%252F%7C1508457929; __tasessionId=tsv5875fx1508462747222; tt_webid=6478472404099532302; WEATHER_CITY=%E5%8C%97%E4%BA%AC'''}
    def city(self):
        print('开始测试')
        city_json=requests.get('https://www.toutiao.com/stream/widget/local_weather/city/', headers=self.headers).json()
        for city in city_json['data']:
            print(city,city_json['data'][city])
            if '湖北' in city:
                return city_json['data'][city]
    def tianqi(self,city):
        #对于网址中中文要先UTF8转换编码，然后urllib.request.quote(content) 转为网址中UTF8
        from urllib import request
        api_url = 'https://www.toutiao.com/stream/widget/local_weather/data/?city=%s' % request.quote(city.encode('utf-8'))

        tianqi_json=requests.get(api_url, headers=self.headers).json()['data']
        return tianqi_json
        #关于网页URL 编码转换http://www.jb51.net/article/86560.htm

    def get_as_cp(self):
        # http://www.cnblogs.com/Ryana/p/7294594.html
        import time
        import hashlib
        zz = {}
        now = round(time.time())
        #print (now, '时间')  # 获取计算机时间
        e = hex(int(now)).upper()[2:]  # hex()转换一个整数对象为十六进制的字符串表示
        #print (e)
        i = hashlib.md5(str(int(now)).encode('utf-8')).hexdigest().upper()  # hashlib.md5().hexdigest()创建hash对象并返回16进制结果

        if len(e) != 8:
            zz = {'as': "479BB4B7254C150",
                  'cp': "7E0AC8874BB0985"}
            return zz
        n = i[:5]
        a = i[-5:]
        r = ""
        s = ""
        for i in range(5):
            s = s + n[i] + e[i]
        for j in range(5):
            r = r + e[j + 3] + a[j]
        zz = {
            'as': "A1" + s + e[-3:],
            'cp': e[0:3] + r + "E1",
            'time': now
        }
        url = 'https://www.toutiao.com/api/pc/feed/?max_behot_time=%s&category=__all__&utm_source=toutiao&widen=1&tadrequire=true&as=%s&cp=%s' % (
            zz['time'], zz['as'], zz['cp'])
        print(url)
        return url
    def toutiao_min_max(self):

        url=self.get_as_cp()
        tianqi_json = requests.get(url, headers=self.headers).json()['data']
        for img_json in tianqi_json:
            if img_json['label']=='广告':
                print('采集到广告，自动跳过')
            # http://ib.snssdk.com/article/v2/tab_comments/?iid=16382074411&group_id=6478657722119094798&item_id=6478657722119094798&offset=20
#评论地址
            else:
                title = img_json['title']  # 标题
                label = img_json['label']  # 所属关键词
                source = img_json['source']  # 作者
                本文_url = 'http://www.toutiao.com/a' + img_json['group_id'] + '/'  # URl拼接 文章ID
                #media_url = img_json['media_url']  # 该号下面论坛
                group_id=img_json['group_id']
                评论地址_url='http://www.toutiao.com/api/comment/list/?group_id=%s&item_id=%s&offset=0&count=5'%(group_id,group_id)

                #访问评论地址后可以拿到评论数量
                #self.pinglun(评论地址_url,group_id) #循环获取评论
                mysql=[title,label,source,group_id,本文_url,评论地址_url]

                print(self.get_mysql(mysql),title,source,group_id,'本文内容',本文_url,评论地址_url)
                time.sleep(0.3)

        self.db.close()


    def get_text(self,url):
        '/html/body/div/div[2]/div[2]/div[1]/div[2]/div/p[1]'
        response=requests.get(url, headers=self.headers)
        #response.encoding = ('utf-8')
        #print(url,'评论相关',etree.HTML(requests.get(url, headers=self.headers).text).xpath('/html/body/text()'))/html/body/article/div[2]/p[3]
        print(response.text)


    def int_mysql(self):
        self.db = pymysql.connect("localhost", "root", "root", "python", charset='utf8mb4', )
        self.cursor = self.db.cursor()
        self.cursor.execute("DROP TABLE IF EXISTS jinritoutiao")
        sqlbiao = """CREATE TABLE jinritoutiao (
                 标题  CHAR(50) NOT NULL,
                 文章链接 CHAR(200),  
                 评论链接 CHAR(200),
                 作者  CHAR(50),
                 文章ID CHAR(50) 
  )"""
        self.cursor.execute(sqlbiao)

        print('结束配置')

    def get_mysql(self,mysql):
        print(mysql[3])
        sql = "SELECT * FROM jinritoutiao \
               WHERE 文章ID = '%s'" % (mysql[3])
        self.cursor.execute(sql)

        if self.cursor.rowcount !=0:

            return '该文章已经存在数据库'

        sql = "INSERT INTO jinritoutiao(标题, \
        文章链接, 评论链接, 作者,文章ID) \
       VALUES ('%s', '%s', '%s', '%s','%s' )" % \
       (mysql[0],  mysql[4],mysql[5], mysql[2],mysql[3])
        try:
            # 执行sql语句
            self.cursor.execute(sql)
            # 提交到数据库执行
            self.db.commit()
            return '写入成功'
        except:
            # 如果发生错误则回滚
            self.db.rollback()
            return '写入失败'

    def pinglun(self,url,group_id):
        pinglun=requests.get(url, headers=self.headers).json()#获取评论地址的json返回值 并且开始进行解析

        pinglushumu_a = int(pinglun['data']['total'])#获取json返回值，解析其中评论的具体数量

        pinglu_shumu_for=divmod(pinglushumu_a,20)#获取评论页数/5 每5条一页

        a = 0

        for pinglu_shumu_for_a in  range(1,pinglu_shumu_for[0]):#获取评论页数/URL

            pinglu_shumu_for_a=pinglu_shumu_for_a*20

            url='http://www.toutiao.com/api/comment/list/?group_id=%s&item_id=%s&offset=0&count=%d'%(group_id,group_id,pinglu_shumu_for_a)#获取评论页数/URL

            pinglun=requests.get(url, headers=self.headers).json()#重新解析每页评论内容

            pinglunshuju=pinglun['data']['comments']

            for pinglun_A in pinglunshuju:
                a = a + 1

                pinglun_text=pinglun_A['text']#用户评论内容

                pinglun_user_name=pinglun_A['user']['name']#用户姓名

                pinglun_user_id = pinglun_A['user']['user_id']#用户ID

                print('评论：',pinglushumu_a,'当前评论第：',a,pinglun_text,pinglun_user_name,pinglun_user_id,url)
            break

        a = 0
    def run(self):
            print('开始测试')
            self.int_mysql()
            for i in range(1,100):
                #try:
                self.toutiao_min_max()
                #except:
                   # print('本次程序出错，重新执行')
                time.sleep(1)
                print('程序已经执行%s次'%i)
            #self.int_mysql()

            self.db.close()




if __name__ == "__main__":
    toutiao=toutiao()
    toutiao.run()

